{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-06-13T01:33:24.083257Z",
     "start_time": "2025-06-13T01:33:24.072977Z"
    }
   },
   "source": [
    "# import os\n",
    "# import time\n",
    "# import numpy as np\n",
    "# from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# from sklearn.svm import SVC\n",
    "# from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# import re\n",
    "# import nltk\n",
    "# from nltk.corpus import stopwords\n",
    "# from nltk.stem import WordNetLemmatizer\n",
    "# \n",
    "# # 下载必要的NLTK数据\n",
    "# nltk.download('stopwords')\n",
    "# nltk.download('wordnet')\n",
    "# \n",
    "# # 文本预处理函数\n",
    "# def preprocess_text(text):\n",
    "#     # 转换为小写\n",
    "#     text = text.lower()\n",
    "#     # 移除HTML标签\n",
    "#     text = re.sub(r'<.*?>', '', text)\n",
    "#     # 移除标点符号和数字\n",
    "#     text = re.sub(r'[^\\w\\s]', '', text)\n",
    "#     text = re.sub(r'\\d+', '', text)\n",
    "#     # 分词\n",
    "#     tokens = text.split()\n",
    "#     # 移除停用词\n",
    "#     stop_words = set(stopwords.words('english'))\n",
    "#     tokens = [word for word in tokens if word not in stop_words]\n",
    "#     # 词形还原\n",
    "#     lemmatizer = WordNetLemmatizer()\n",
    "#     tokens = [lemmatizer.lemmatize(word) for word in tokens]\n",
    "#     # 重新组合文本\n",
    "#     return ' '.join(tokens)\n",
    "# \n",
    "# # 加载IMDb数据集\n",
    "# def load_imdb_data(data_dir):\n",
    "#     texts = []\n",
    "#     labels = []\n",
    "#     \n",
    "#     # 加载正面评论\n",
    "#     pos_dir = os.path.join(data_dir, 'pos')\n",
    "#     for file in os.listdir(pos_dir):\n",
    "#         with open(os.path.join(pos_dir, file), 'r', encoding='utf-8') as f:\n",
    "#             texts.append(f.read())\n",
    "#             labels.append(1)  # 正面评论标记为1\n",
    "#     \n",
    "#     # 加载负面评论\n",
    "#     neg_dir = os.path.join(data_dir, 'neg')\n",
    "#     for file in os.listdir(neg_dir):\n",
    "#         with open(os.path.join(neg_dir, file), 'r', encoding='utf-8') as f:\n",
    "#             texts.append(f.read())\n",
    "#             labels.append(0)  # 负面评论标记为0\n",
    "#     \n",
    "#     return texts, labels\n",
    "# \n",
    "# # 主函数\n",
    "# def main():\n",
    "#     # 替换为您的数据路径\n",
    "#     train_data_dir = '../Data/aclImdb_v1/aclImdb/train'\n",
    "#     test_data_dir = '../Data/aclImdb_v1/aclImdb/test'\n",
    "#     \n",
    "#     print(\"加载训练数据...\")\n",
    "#     train_texts, train_labels = load_imdb_data(train_data_dir)\n",
    "#     \n",
    "#     print(\"加载测试数据...\")\n",
    "#     test_texts, test_labels = load_imdb_data(test_data_dir)\n",
    "#     \n",
    "#     print(\"数据预处理...\")\n",
    "#     train_texts_processed = [preprocess_text(text) for text in train_texts]\n",
    "#     test_texts_processed = [preprocess_text(text) for text in test_texts]\n",
    "#     \n",
    "#     print(\"特征提取...\")\n",
    "#     vectorizer = TfidfVectorizer(max_features=5000)\n",
    "#     X_train = vectorizer.fit_transform(train_texts_processed)\n",
    "#     X_test = vectorizer.transform(test_texts_processed)\n",
    "#     \n",
    "#     print(\"训练SVM模型...\")\n",
    "#     start_time = time.time()\n",
    "#     svm_model = SVC(kernel='linear')\n",
    "#     svm_model.fit(X_train, train_labels)\n",
    "#     training_time = time.time() - start_time\n",
    "#     \n",
    "#     print(\"评估模型...\")\n",
    "#     y_pred = svm_model.predict(X_test)\n",
    "#     \n",
    "#     # 计算评估指标\n",
    "#     accuracy = accuracy_score(test_labels, y_pred)\n",
    "#     precision = precision_score(test_labels, y_pred)\n",
    "#     recall = recall_score(test_labels, y_pred)\n",
    "#     f1 = f1_score(test_labels, y_pred)\n",
    "#     \n",
    "#     # 打印结果\n",
    "#     print(f\"训练时间: {training_time:.2f}秒\")\n",
    "#     print(f\"准确率: {accuracy:.4f}\")\n",
    "#     print(f\"精确率: {precision:.4f}\")\n",
    "#     print(f\"召回率: {recall:.4f}\")\n",
    "#     print(f\"F1值: {f1:.4f}\")\n",
    "# \n",
    "# if __name__ == \"__main__\":\n",
    "#     main()\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-12T09:46:37.943775Z",
     "start_time": "2025-06-12T09:34:23.728392Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import time\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "class SVM:\n",
    "    def __init__(self, learning_rate=0.001, lambda_param=0.01, n_iters=1000):\n",
    "        self.lr = learning_rate\n",
    "        self.lambda_param = lambda_param\n",
    "        self.n_iters = n_iters\n",
    "        self.w = None\n",
    "        self.b = None\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        n_samples, n_features = X.shape\n",
    "        \n",
    "        # 将标签转换为-1和1\n",
    "        y_ = np.where(y <= 0, -1, 1)\n",
    "        \n",
    "        # 初始化权重和偏置\n",
    "        self.w = np.zeros(n_features)\n",
    "        self.b = 0\n",
    "        \n",
    "        # 梯度下降训练\n",
    "        for _ in range(self.n_iters):\n",
    "            for idx, x_i in enumerate(X):\n",
    "                condition = y_[idx] * (np.dot(x_i, self.w) - self.b) >= 1\n",
    "                if condition:\n",
    "                    self.w -= self.lr * (2 * self.lambda_param * self.w)\n",
    "                else:\n",
    "                    self.w -= self.lr * (2 * self.lambda_param * self.w - np.dot(x_i, y_[idx]))\n",
    "                    self.b -= self.lr * y_[idx]\n",
    "\n",
    "    def predict(self, X):\n",
    "        approx = np.dot(X, self.w) - self.b\n",
    "        return np.sign(approx)\n",
    "\n",
    "def load_imdb_data(data_dir):\n",
    "    \"\"\"加载IMDb数据集\"\"\"\n",
    "    reviews = []\n",
    "    labels = []\n",
    "    \n",
    "    # 加载正面评论\n",
    "    pos_dir = os.path.join(data_dir, 'pos')\n",
    "    for file in os.listdir(pos_dir):\n",
    "        with open(os.path.join(pos_dir, file), 'r', encoding='utf-8') as f:\n",
    "            reviews.append(f.read())\n",
    "            labels.append(1)  # 正面评论标记为1\n",
    "    \n",
    "    # 加载负面评论\n",
    "    neg_dir = os.path.join(data_dir, 'neg')\n",
    "    for file in os.listdir(neg_dir):\n",
    "        with open(os.path.join(neg_dir, file), 'r', encoding='utf-8') as f:\n",
    "            reviews.append(f.read())\n",
    "            labels.append(0)  # 负面评论标记为0\n",
    "    \n",
    "    return reviews, labels\n",
    "\n",
    "def preprocess_text(text):\n",
    "    \"\"\"简单的文本预处理\"\"\"\n",
    "    # 移除HTML标签\n",
    "    text = re.sub(r'<.*?>', '', text)\n",
    "    # 移除特殊字符\n",
    "    text = re.sub(r'[^a-zA-Z\\s]', '', text)\n",
    "    # 转换为小写\n",
    "    text = text.lower()\n",
    "    return text\n",
    "\n",
    "def main():\n",
    "    # 设置数据路径\n",
    "    data_path = '../Data/aclImdb_v1/aclImdb/train'  # 请替换为实际路径\n",
    "    \n",
    "    print(\"加载数据...\")\n",
    "    reviews, labels = load_imdb_data(data_path)\n",
    "    \n",
    "    print(\"预处理文本...\")\n",
    "    preprocessed_reviews = [preprocess_text(review) for review in reviews]\n",
    "    \n",
    "    print(\"特征提取...\")\n",
    "    vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')\n",
    "    X = vectorizer.fit_transform(preprocessed_reviews).toarray()\n",
    "    \n",
    "    print(\"数据标准化...\")\n",
    "    scaler = StandardScaler()\n",
    "    X = scaler.fit_transform(X)\n",
    "    \n",
    "    print(\"划分训练集和测试集...\")\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.2, random_state=42)\n",
    "    \n",
    "    print(\"训练SVM模型...\")\n",
    "    start_time = time.time()\n",
    "    svm = SVM(learning_rate=0.001, lambda_param=0.01, n_iters=1000)\n",
    "    svm.fit(X_train, np.array(y_train))\n",
    "    training_time = time.time() - start_time\n",
    "    \n",
    "    print(\"模型评估...\")\n",
    "    y_pred = svm.predict(X_test)\n",
    "    # 将预测结果从-1/1转换回0/1\n",
    "    y_pred = np.where(y_pred == -1, 0, 1)\n",
    "    \n",
    "    # 计算评估指标\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    precision = precision_score(y_test, y_pred)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    \n",
    "    print(f\"准确率: {accuracy:.4f}\")\n",
    "    print(f\"精确率: {precision:.4f}\")\n",
    "    print(f\"召回率: {recall:.4f}\")\n",
    "    print(f\"F1值: {f1:.4f}\")\n",
    "    print(f\"训练时间: {training_time:.2f}秒\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ],
   "id": "de73de8d605311cb",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "加载数据...\n",
      "预处理文本...\n",
      "特征提取...\n",
      "数据标准化...\n",
      "划分训练集和测试集...\n",
      "训练SVM模型...\n",
      "模型评估...\n",
      "准确率: 0.8330\n",
      "精确率: 0.8271\n",
      "召回率: 0.8445\n",
      "F1值: 0.8357\n",
      "训练时间: 617.63秒\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
